server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityHashTagObject,
   7   ActivityMagnetUrlObject,
   8   ActivityPlaylistSegmentHashesObject,
   9   ActivityPlaylistUrlObject, ActivitypubHttpFetcherPayload,
  10   ActivityTagObject,
  11   ActivityUrlObject,
  12   ActivityVideoUrlObject,
  13   VideoState
  14 } from '../../../shared/index'
  15 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  16 import { VideoPrivacy } from '../../../shared/models/videos'
  17 import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  18 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  19 import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  20 import { logger } from '../../helpers/logger'
  21 import { doRequest } from '../../helpers/requests'
  22 import {
  23   ACTIVITY_PUB,
  24   MIMETYPES,
  25   P2P_MEDIA_LOADER_PEER_VERSION,
  26   PREVIEWS_SIZE,
  27   REMOTE_SCHEME,
  28   STATIC_PATHS,
  29   THUMBNAILS_SIZE
  30 } from '../../initializers/constants'
  31 import { TagModel } from '../../models/video/tag'
  32 import { VideoModel } from '../../models/video/video'
  33 import { VideoFileModel } from '../../models/video/video-file'
  34 import { getOrCreateActorAndServerAndModel } from './actor'
  35 import { addVideoComments } from './video-comments'
  36 import { crawlCollectionPage } from './crawl'
  37 import { sendCreateVideo, sendUpdateVideo } from './send'
  38 import { isArray } from '../../helpers/custom-validators/misc'
  39 import { VideoCaptionModel } from '../../models/video/video-caption'
  40 import { JobQueue } from '../job-queue'
  41 import { createRates } from './video-rates'
  42 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  43 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  44 import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  45 import { Notifier } from '../notifier'
  46 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  47 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  48 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  49 import { VideoShareModel } from '../../models/video/video-share'
  50 import { VideoCommentModel } from '../../models/video/video-comment'
  51 import { sequelizeTypescript } from '../../initializers/database'
  52 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
  53 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  54 import { join } from 'path'
  55 import { FilteredModelAttributes } from '../../typings/sequelize'
  56 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
  57 import { ActorFollowScoreCache } from '../files-cache'
  58 import {
  59   MAccountIdActor,
  60   MChannelAccountLight,
  61   MChannelDefault,
  62   MChannelId,
  63   MStreamingPlaylist,
  64   MVideo,
  65   MVideoAccountLight,
  66   MVideoAccountLightBlacklistAllFiles,
  67   MVideoAP,
  68   MVideoAPWithoutCaption,
  69   MVideoFile,
  70   MVideoFullLight,
  71   MVideoId,
  72   MVideoImmutable,
  73   MVideoThumbnail
  74 } from '../../typings/models'
  75 import { MThumbnail } from '../../typings/models/video/thumbnail'
  76 import { maxBy, minBy } from 'lodash'
  77
  78 async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  79   const video = videoArg as MVideoAP
  80
  81   if (
  82     // Check this is not a blacklisted video, or unfederated blacklisted video
  83     (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
  84     // Check the video is public/unlisted and published
  85     video.hasPrivacyForFederation() && video.state === VideoState.PUBLISHED
  86   ) {
  87     // Fetch more attributes that we will need to serialize in AP object
  88     if (isArray(video.VideoCaptions) === false) {
  89       video.VideoCaptions = await video.$get('VideoCaptions', {
  90         attributes: [ 'language' ],
  91         transaction
  92       })
  93     }
  94
  95     if (isNewVideo) {
  96       // Now we'll add the video's meta data to our followers
  97       await sendCreateVideo(video, transaction)
  98       await shareVideoByServerAndChannel(video, transaction)
  99     } else {
 100       await sendUpdateVideo(video, transaction)
 101     }
 102   }
 103 }
 104
 105 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
 106   const options = {
 107     uri: videoUrl,
 108     method: 'GET',
 109     json: true,
 110     activityPub: true
 111   }
 112
 113   logger.info('Fetching remote video %s.', videoUrl)
 114
 115   const { response, body } = await doRequest<any>(options)
 116
 117   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
 118     logger.debug('Remote video JSON is not valid.', { body })
 119     return { response, videoObject: undefined }
 120   }
 121
 122   return { response, videoObject: body }
 123 }
 124
 125 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
 126   const host = video.VideoChannel.Account.Actor.Server.host
 127   const path = video.getDescriptionAPIPath()
 128   const options = {
 129     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 130     json: true
 131   }
 132
 133   const { body } = await doRequest<any>(options)
 134   return body.description ? body.description : ''
 135 }
 136
 137 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 138   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 139   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 140
 141   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 142     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 143   }
 144
 145   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 146 }
 147
 148 type SyncParam = {
 149   likes: boolean
 150   dislikes: boolean
 151   shares: boolean
 152   comments: boolean
 153   thumbnail: boolean
 154   refreshVideo?: boolean
 155 }
 156 async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 157   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 158
 159   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 160
 161   if (syncParam.likes === true) {
 162     const handler = items => createRates(items, video, 'like')
 163     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 164
 165     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 166       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
 167   } else {
 168     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 169   }
 170
 171   if (syncParam.dislikes === true) {
 172     const handler = items => createRates(items, video, 'dislike')
 173     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 174
 175     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 176       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
 177   } else {
 178     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 179   }
 180
 181   if (syncParam.shares === true) {
 182     const handler = items => addVideoShares(items, video)
 183     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 184
 185     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 186       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
 187   } else {
 188     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 189   }
 190
 191   if (syncParam.comments === true) {
 192     const handler = items => addVideoComments(items)
 193     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 194
 195     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 196       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
 197   } else {
 198     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 199   }
 200
 201   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
 202 }
 203
 204 type GetVideoResult <T> = Promise<{
 205   video: T
 206   created: boolean
 207   autoBlacklisted?: boolean
 208 }>
 209
 210 type GetVideoParamAll = {
 211   videoObject: { id: string } | string
 212   syncParam?: SyncParam
 213   fetchType?: 'all'
 214   allowRefresh?: boolean
 215 }
 216
 217 type GetVideoParamImmutable = {
 218   videoObject: { id: string } | string
 219   syncParam?: SyncParam
 220   fetchType: 'only-immutable-attributes'
 221   allowRefresh: false
 222 }
 223
 224 type GetVideoParamOther = {
 225   videoObject: { id: string } | string
 226   syncParam?: SyncParam
 227   fetchType?: 'all' | 'only-video'
 228   allowRefresh?: boolean
 229 }
 230
 231 function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
 232 function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
 233 function getOrCreateVideoAndAccountAndChannel (
 234   options: GetVideoParamOther
 235 ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
 236 async function getOrCreateVideoAndAccountAndChannel (
 237   options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
 238 ): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
 239   // Default params
 240   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 241   const fetchType = options.fetchType || 'all'
 242   const allowRefresh = options.allowRefresh !== false
 243
 244   // Get video url
 245   const videoUrl = getAPId(options.videoObject)
 246   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 247
 248   if (videoFromDatabase) {
 249     // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
 250     if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
 251       const refreshOptions = {
 252         video: videoFromDatabase as MVideoThumbnail,
 253         fetchedType: fetchType,
 254         syncParam
 255       }
 256
 257       if (syncParam.refreshVideo === true) {
 258         videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 259       } else {
 260         await JobQueue.Instance.createJobWithPromise({
 261           type: 'activitypub-refresher',
 262           payload: { type: 'video', url: videoFromDatabase.url }
 263         })
 264       }
 265     }
 266
 267     return { video: videoFromDatabase, created: false }
 268   }
 269
 270   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 271   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 272
 273   const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 274   const videoChannel = actor.VideoChannel
 275
 276   try {
 277     const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
 278
 279     await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
 280
 281     return { video: videoCreated, created: true, autoBlacklisted }
 282   } catch (err) {
 283     // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
 284     if (err.name === 'SequelizeUniqueConstraintError') {
 285       const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
 286       if (fallbackVideo) return { video: fallbackVideo, created: false }
 287     }
 288
 289     throw err
 290   }
 291 }
 292
 293 async function updateVideoFromAP (options: {
 294   video: MVideoAccountLightBlacklistAllFiles
 295   videoObject: VideoTorrentObject
 296   account: MAccountIdActor
 297   channel: MChannelDefault
 298   overrideTo?: string[]
 299 }) {
 300   const { video, videoObject, account, channel, overrideTo } = options
 301
 302   logger.debug('Updating remote video "%s".', options.videoObject.uuid, { account, channel })
 303
 304   let videoFieldsSave: any
 305   const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
 306   const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
 307
 308   try {
 309     let thumbnailModel: MThumbnail
 310
 311     try {
 312       thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
 313     } catch (err) {
 314       logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
 315     }
 316
 317     const videoUpdated = await sequelizeTypescript.transaction(async t => {
 318       const sequelizeOptions = { transaction: t }
 319
 320       videoFieldsSave = video.toJSON()
 321
 322       // Check actor has the right to update the video
 323       const videoChannel = video.VideoChannel
 324       if (videoChannel.Account.id !== account.id) {
 325         throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 326       }
 327
 328       const to = overrideTo || videoObject.to
 329       const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
 330       video.name = videoData.name
 331       video.uuid = videoData.uuid
 332       video.url = videoData.url
 333       video.category = videoData.category
 334       video.licence = videoData.licence
 335       video.language = videoData.language
 336       video.description = videoData.description
 337       video.support = videoData.support
 338       video.nsfw = videoData.nsfw
 339       video.commentsEnabled = videoData.commentsEnabled
 340       video.downloadEnabled = videoData.downloadEnabled
 341       video.waitTranscoding = videoData.waitTranscoding
 342       video.state = videoData.state
 343       video.duration = videoData.duration
 344       video.createdAt = videoData.createdAt
 345       video.publishedAt = videoData.publishedAt
 346       video.originallyPublishedAt = videoData.originallyPublishedAt
 347       video.privacy = videoData.privacy
 348       video.channelId = videoData.channelId
 349       video.views = videoData.views
 350
 351       const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
 352
 353       if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
 354
 355       if (videoUpdated.getPreview()) {
 356         const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated)
 357         const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 358         await videoUpdated.addAndSaveThumbnail(previewModel, t)
 359       }
 360
 361       {
 362         const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
 363         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 364
 365         // Remove video files that do not exist anymore
 366         const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
 367         await Promise.all(destroyTasks)
 368
 369         // Update or add other one
 370         const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
 371         videoUpdated.VideoFiles = await Promise.all(upsertTasks)
 372       }
 373
 374       {
 375         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
 376         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 377
 378         // Remove video playlists that do not exist anymore
 379         const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
 380         await Promise.all(destroyTasks)
 381
 382         let oldStreamingPlaylistFiles: MVideoFile[] = []
 383         for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
 384           oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
 385         }
 386
 387         videoUpdated.VideoStreamingPlaylists = []
 388
 389         for (const playlistAttributes of streamingPlaylistAttributes) {
 390           const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
 391                                      .then(([ streamingPlaylist ]) => streamingPlaylist)
 392
 393           const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
 394             .map(a => new VideoFileModel(a))
 395           const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
 396           await Promise.all(destroyTasks)
 397
 398           // Update or add other one
 399           const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
 400           streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
 401
 402           videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
 403         }
 404       }
 405
 406       {
 407         // Update Tags
 408         const tags = videoObject.tag
 409                                 .filter(isAPHashTagObject)
 410                                 .map(tag => tag.name)
 411         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 412         await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
 413       }
 414
 415       {
 416         // Update captions
 417         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
 418
 419         const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 420           return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, c.url, t)
 421         })
 422         await Promise.all(videoCaptionsPromises)
 423       }
 424
 425       return videoUpdated
 426     })
 427
 428     await autoBlacklistVideoIfNeeded({
 429       video: videoUpdated,
 430       user: undefined,
 431       isRemote: true,
 432       isNew: false,
 433       transaction: undefined
 434     })
 435
 436     if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) // Notify our users?
 437
 438     logger.info('Remote video with uuid %s updated', videoObject.uuid)
 439
 440     return videoUpdated
 441   } catch (err) {
 442     if (video !== undefined && videoFieldsSave !== undefined) {
 443       resetSequelizeInstance(video, videoFieldsSave)
 444     }
 445
 446     // This is just a debug because we will retry the insert
 447     logger.debug('Cannot update the remote video.', { err })
 448     throw err
 449   }
 450 }
 451
 452 async function refreshVideoIfNeeded (options: {
 453   video: MVideoThumbnail
 454   fetchedType: VideoFetchByUrlType
 455   syncParam: SyncParam
 456 }): Promise<MVideoThumbnail> {
 457   if (!options.video.isOutdated()) return options.video
 458
 459   // We need more attributes if the argument video was fetched with not enough joints
 460   const video = options.fetchedType === 'all'
 461     ? options.video as MVideoAccountLightBlacklistAllFiles
 462     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 463
 464   try {
 465     const { response, videoObject } = await fetchRemoteVideo(video.url)
 466     if (response.statusCode === 404) {
 467       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 468
 469       // Video does not exist anymore
 470       await video.destroy()
 471       return undefined
 472     }
 473
 474     if (videoObject === undefined) {
 475       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 476
 477       await video.setAsRefreshed()
 478       return video
 479     }
 480
 481     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 482
 483     const updateOptions = {
 484       video,
 485       videoObject,
 486       account: channelActor.VideoChannel.Account,
 487       channel: channelActor.VideoChannel
 488     }
 489     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 490     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 491
 492     ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
 493
 494     return video
 495   } catch (err) {
 496     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 497
 498     ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
 499
 500     // Don't refresh in loop
 501     await video.setAsRefreshed()
 502     return video
 503   }
 504 }
 505
 506 export {
 507   updateVideoFromAP,
 508   refreshVideoIfNeeded,
 509   federateVideoIfNeeded,
 510   fetchRemoteVideo,
 511   getOrCreateVideoAndAccountAndChannel,
 512   fetchRemoteVideoDescription,
 513   getOrCreateVideoChannelFromVideoObject
 514 }
 515
 516 // ---------------------------------------------------------------------------
 517
 518 function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
 519   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 520
 521   const urlMediaType = url.mediaType
 522   return mimeTypes.includes(urlMediaType) && urlMediaType.startsWith('video/')
 523 }
 524
 525 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 526   return url && url.mediaType === 'application/x-mpegURL'
 527 }
 528
 529 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 530   return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
 531 }
 532
 533 function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
 534   return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
 535 }
 536
 537 function isAPHashTagObject (url: any): url is ActivityHashTagObject {
 538   return url && url.type === 'Hashtag'
 539 }
 540
 541 async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
 542   logger.debug('Adding remote video %s.', videoObject.id)
 543
 544   const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
 545   const video = VideoModel.build(videoData) as MVideoThumbnail
 546
 547   const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE)
 548     .catch(err => {
 549       logger.error('Cannot create miniature from url.', { err })
 550       return undefined
 551     })
 552
 553   let thumbnailModel: MThumbnail
 554   if (waitThumbnail === true) {
 555     thumbnailModel = await promiseThumbnail
 556   }
 557
 558   const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
 559     const sequelizeOptions = { transaction: t }
 560
 561     const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
 562     videoCreated.VideoChannel = channel
 563
 564     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 565
 566     const previewIcon = getPreviewFromIcons(videoObject)
 567     const previewUrl = previewIcon
 568       ? previewIcon.url
 569       : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 570     const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 571
 572     if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
 573
 574     // Process files
 575     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
 576
 577     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 578     const videoFiles = await Promise.all(videoFilePromises)
 579
 580     const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 581     videoCreated.VideoStreamingPlaylists = []
 582
 583     for (const playlistAttributes of streamingPlaylistsAttributes) {
 584       const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t })
 585
 586       const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject)
 587       const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
 588       playlistModel.VideoFiles = await Promise.all(videoFilePromises)
 589
 590       videoCreated.VideoStreamingPlaylists.push(playlistModel)
 591     }
 592
 593     // Process tags
 594     const tags = videoObject.tag
 595                             .filter(isAPHashTagObject)
 596                             .map(t => t.name)
 597     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 598     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 599
 600     // Process captions
 601     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 602       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, c.url, t)
 603     })
 604     await Promise.all(videoCaptionsPromises)
 605
 606     videoCreated.VideoFiles = videoFiles
 607     videoCreated.Tags = tagInstances
 608
 609     const autoBlacklisted = await autoBlacklistVideoIfNeeded({
 610       video: videoCreated,
 611       user: undefined,
 612       isRemote: true,
 613       isNew: true,
 614       transaction: t
 615     })
 616
 617     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 618
 619     return { autoBlacklisted, videoCreated }
 620   })
 621
 622   if (waitThumbnail === false) {
 623     // Error is already caught above
 624     // eslint-disable-next-line @typescript-eslint/no-floating-promises
 625     promiseThumbnail.then(thumbnailModel => {
 626       if (!thumbnailModel) return
 627
 628       thumbnailModel = videoCreated.id
 629
 630       return thumbnailModel.save()
 631     })
 632   }
 633
 634   return { autoBlacklisted, videoCreated }
 635 }
 636
 637 function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) {
 638   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
 639     ? VideoPrivacy.PUBLIC
 640     : VideoPrivacy.UNLISTED
 641
 642   const duration = videoObject.duration.replace(/[^\d]+/, '')
 643   const language = videoObject.language?.identifier
 644
 645   const category = videoObject.category
 646     ? parseInt(videoObject.category.identifier, 10)
 647     : undefined
 648
 649   const licence = videoObject.licence
 650     ? parseInt(videoObject.licence.identifier, 10)
 651     : undefined
 652
 653   const description = videoObject.content || null
 654   const support = videoObject.support || null
 655
 656   return {
 657     name: videoObject.name,
 658     uuid: videoObject.uuid,
 659     url: videoObject.id,
 660     category,
 661     licence,
 662     language,
 663     description,
 664     support,
 665     nsfw: videoObject.sensitive,
 666     commentsEnabled: videoObject.commentsEnabled,
 667     downloadEnabled: videoObject.downloadEnabled,
 668     waitTranscoding: videoObject.waitTranscoding,
 669     state: videoObject.state,
 670     channelId: videoChannel.id,
 671     duration: parseInt(duration, 10),
 672     createdAt: new Date(videoObject.published),
 673     publishedAt: new Date(videoObject.published),
 674
 675     originallyPublishedAt: videoObject.originallyPublishedAt
 676       ? new Date(videoObject.originallyPublishedAt)
 677       : null,
 678
 679     updatedAt: new Date(videoObject.updated),
 680     views: videoObject.views,
 681     likes: 0,
 682     dislikes: 0,
 683     remote: true,
 684     privacy
 685   }
 686 }
 687
 688 function videoFileActivityUrlToDBAttributes (
 689   videoOrPlaylist: MVideo | MStreamingPlaylist,
 690   urls: (ActivityTagObject | ActivityUrlObject)[]
 691 ) {
 692   const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 693
 694   if (fileUrls.length === 0) return []
 695
 696   const attributes: FilteredModelAttributes<VideoFileModel>[] = []
 697   for (const fileUrl of fileUrls) {
 698     // Fetch associated magnet uri
 699     const magnet = urls.filter(isAPMagnetUrlObject)
 700                        .find(u => u.height === fileUrl.height)
 701
 702     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 703
 704     const parsed = magnetUtil.decode(magnet.href)
 705     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 706       throw new Error('Cannot parse magnet URI ' + magnet.href)
 707     }
 708
 709     // Fetch associated metadata url, if any
 710     const metadata = urls.filter(isAPVideoFileMetadataObject)
 711                          .find(u => {
 712                            return u.height === fileUrl.height &&
 713                              u.fps === fileUrl.fps &&
 714                              u.rel.includes(fileUrl.mediaType)
 715                          })
 716
 717     const mediaType = fileUrl.mediaType
 718     const attribute = {
 719       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType],
 720       infoHash: parsed.infoHash,
 721       resolution: fileUrl.height,
 722       size: fileUrl.size,
 723       fps: fileUrl.fps || -1,
 724       metadataUrl: metadata?.href,
 725
 726       // This is a video file owned by a video or by a streaming playlist
 727       videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,
 728       videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
 729     }
 730
 731     attributes.push(attribute)
 732   }
 733
 734   return attributes
 735 }
 736
 737 function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoTorrentObject, videoFiles: MVideoFile[]) {
 738   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 739   if (playlistUrls.length === 0) return []
 740
 741   const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
 742   for (const playlistUrlObject of playlistUrls) {
 743     const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
 744
 745     let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 746
 747     // FIXME: backward compatibility introduced in v2.1.0
 748     if (files.length === 0) files = videoFiles
 749
 750     if (!segmentsSha256UrlObject) {
 751       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 752       continue
 753     }
 754
 755     const attribute = {
 756       type: VideoStreamingPlaylistType.HLS,
 757       playlistUrl: playlistUrlObject.href,
 758       segmentsSha256Url: segmentsSha256UrlObject.href,
 759       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
 760       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 761       videoId: video.id,
 762       tagAPObject: playlistUrlObject.tag
 763     }
 764
 765     attributes.push(attribute)
 766   }
 767
 768   return attributes
 769 }
 770
 771 function getThumbnailFromIcons (videoObject: VideoTorrentObject) {
 772   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
 773   // Fallback if there are not valid icons
 774   if (validIcons.length === 0) validIcons = videoObject.icon
 775
 776   return minBy(validIcons, 'width')
 777 }
 778
 779 function getPreviewFromIcons (videoObject: VideoTorrentObject) {
 780   const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
 781
 782   // FIXME: don't put a fallback here for compatibility with PeerTube <2.2
 783
 784   return maxBy(validIcons, 'width')
 785 }