server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityPlaylistSegmentHashesObject,
   7   ActivityPlaylistUrlObject,
   8   ActivityUrlObject,
   9   ActivityVideoUrlObject, VideoCreate,
  10   VideoState
  11 } from '../../../shared/index'
  12 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  13 import { VideoPrivacy } from '../../../shared/models/videos'
  14 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  15 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  16 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  17 import { logger } from '../../helpers/logger'
  18 import { doRequest } from '../../helpers/requests'
  19 import {
  20   ACTIVITY_PUB,
  21   MIMETYPES,
  22   P2P_MEDIA_LOADER_PEER_VERSION,
  23   PREVIEWS_SIZE,
  24   REMOTE_SCHEME,
  25   STATIC_PATHS
  26 } from '../../initializers/constants'
  27 import { ActorModel } from '../../models/activitypub/actor'
  28 import { TagModel } from '../../models/video/tag'
  29 import { VideoModel } from '../../models/video/video'
  30 import { VideoChannelModel } from '../../models/video/video-channel'
  31 import { VideoFileModel } from '../../models/video/video-file'
  32 import { getOrCreateActorAndServerAndModel } from './actor'
  33 import { addVideoComments } from './video-comments'
  34 import { crawlCollectionPage } from './crawl'
  35 import { sendCreateVideo, sendUpdateVideo } from './send'
  36 import { isArray } from '../../helpers/custom-validators/misc'
  37 import { VideoCaptionModel } from '../../models/video/video-caption'
  38 import { JobQueue } from '../job-queue'
  39 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
  40 import { createRates } from './video-rates'
  41 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  42 import { AccountModel } from '../../models/account/account'
  43 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  44 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  45 import { Notifier } from '../notifier'
  46 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  47 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  48 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
  49 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  50 import { VideoShareModel } from '../../models/video/video-share'
  51 import { VideoCommentModel } from '../../models/video/video-comment'
  52 import { sequelizeTypescript } from '../../initializers/database'
  53 import { createPlaceholderThumbnail, createVideoThumbnailFromUrl } from '../thumbnail'
  54 import { ThumbnailModel } from '../../models/video/thumbnail'
  55 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  56 import { join } from 'path'
  57
  58 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  59   // If the video is not private and is published, we federate it
  60   if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
  61     // Fetch more attributes that we will need to serialize in AP object
  62     if (isArray(video.VideoCaptions) === false) {
  63       video.VideoCaptions = await video.$get('VideoCaptions', {
  64         attributes: [ 'language' ],
  65         transaction
  66       }) as VideoCaptionModel[]
  67     }
  68
  69     if (isNewVideo) {
  70       // Now we'll add the video's meta data to our followers
  71       await sendCreateVideo(video, transaction)
  72       await shareVideoByServerAndChannel(video, transaction)
  73     } else {
  74       await sendUpdateVideo(video, transaction)
  75     }
  76   }
  77 }
  78
  79 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
  80   const options = {
  81     uri: videoUrl,
  82     method: 'GET',
  83     json: true,
  84     activityPub: true
  85   }
  86
  87   logger.info('Fetching remote video %s.', videoUrl)
  88
  89   const { response, body } = await doRequest(options)
  90
  91   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
  92     logger.debug('Remote video JSON is not valid.', { body })
  93     return { response, videoObject: undefined }
  94   }
  95
  96   return { response, videoObject: body }
  97 }
  98
  99 async function fetchRemoteVideoDescription (video: VideoModel) {
 100   const host = video.VideoChannel.Account.Actor.Server.host
 101   const path = video.getDescriptionAPIPath()
 102   const options = {
 103     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 104     json: true
 105   }
 106
 107   const { body } = await doRequest(options)
 108   return body.description ? body.description : ''
 109 }
 110
 111 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
 112   const url = buildRemoteBaseUrl(video, path)
 113
 114   // We need to provide a callback, if no we could have an uncaught exception
 115   return request.get(url, err => {
 116     if (err) reject(err)
 117   })
 118 }
 119
 120 function buildRemoteBaseUrl (video: VideoModel, path: string) {
 121   const host = video.VideoChannel.Account.Actor.Server.host
 122
 123   return REMOTE_SCHEME.HTTP + '://' + host + path
 124 }
 125
 126 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 127   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 128   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 129
 130   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 131     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 132   }
 133
 134   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 135 }
 136
 137 type SyncParam = {
 138   likes: boolean
 139   dislikes: boolean
 140   shares: boolean
 141   comments: boolean
 142   thumbnail: boolean
 143   refreshVideo?: boolean
 144 }
 145 async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 146   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 147
 148   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 149
 150   if (syncParam.likes === true) {
 151     const handler = items => createRates(items, video, 'like')
 152     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 153
 154     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 155       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
 156   } else {
 157     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 158   }
 159
 160   if (syncParam.dislikes === true) {
 161     const handler = items => createRates(items, video, 'dislike')
 162     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 163
 164     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 165       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
 166   } else {
 167     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 168   }
 169
 170   if (syncParam.shares === true) {
 171     const handler = items => addVideoShares(items, video)
 172     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 173
 174     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 175       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
 176   } else {
 177     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 178   }
 179
 180   if (syncParam.comments === true) {
 181     const handler = items => addVideoComments(items, video)
 182     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 183
 184     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 185       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
 186   } else {
 187     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 188   }
 189
 190   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
 191 }
 192
 193 async function getOrCreateVideoAndAccountAndChannel (options: {
 194   videoObject: { id: string } | string,
 195   syncParam?: SyncParam,
 196   fetchType?: VideoFetchByUrlType,
 197   allowRefresh?: boolean // true by default
 198 }) {
 199   // Default params
 200   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 201   const fetchType = options.fetchType || 'all'
 202   const allowRefresh = options.allowRefresh !== false
 203
 204   // Get video url
 205   const videoUrl = getAPId(options.videoObject)
 206
 207   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 208   if (videoFromDatabase) {
 209     if (videoFromDatabase.isOutdated() && allowRefresh === true) {
 210       const refreshOptions = {
 211         video: videoFromDatabase,
 212         fetchedType: fetchType,
 213         syncParam
 214       }
 215
 216       if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 217       else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
 218     }
 219
 220     return { video: videoFromDatabase, created: false }
 221   }
 222
 223   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 224   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 225
 226   const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 227   const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
 228
 229   await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
 230
 231   return { video, created: true }
 232 }
 233
 234 async function updateVideoFromAP (options: {
 235   video: VideoModel,
 236   videoObject: VideoTorrentObject,
 237   account: AccountModel,
 238   channel: VideoChannelModel,
 239   overrideTo?: string[]
 240 }) {
 241   logger.debug('Updating remote video "%s".', options.videoObject.uuid)
 242
 243   let videoFieldsSave: any
 244   const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE
 245   const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
 246
 247   try {
 248     let thumbnailModel: ThumbnailModel
 249
 250     try {
 251       thumbnailModel = await createVideoThumbnailFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.THUMBNAIL)
 252     } catch (err) {
 253       logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
 254     }
 255
 256     await sequelizeTypescript.transaction(async t => {
 257       const sequelizeOptions = { transaction: t }
 258
 259       videoFieldsSave = options.video.toJSON()
 260
 261       // Check actor has the right to update the video
 262       const videoChannel = options.video.VideoChannel
 263       if (videoChannel.Account.id !== options.account.id) {
 264         throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 265       }
 266
 267       const to = options.overrideTo ? options.overrideTo : options.videoObject.to
 268       const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
 269       options.video.set('name', videoData.name)
 270       options.video.set('uuid', videoData.uuid)
 271       options.video.set('url', videoData.url)
 272       options.video.set('category', videoData.category)
 273       options.video.set('licence', videoData.licence)
 274       options.video.set('language', videoData.language)
 275       options.video.set('description', videoData.description)
 276       options.video.set('support', videoData.support)
 277       options.video.set('nsfw', videoData.nsfw)
 278       options.video.set('commentsEnabled', videoData.commentsEnabled)
 279       options.video.set('downloadEnabled', videoData.downloadEnabled)
 280       options.video.set('waitTranscoding', videoData.waitTranscoding)
 281       options.video.set('state', videoData.state)
 282       options.video.set('duration', videoData.duration)
 283       options.video.set('createdAt', videoData.createdAt)
 284       options.video.set('publishedAt', videoData.publishedAt)
 285       options.video.set('originallyPublishedAt', videoData.originallyPublishedAt)
 286       options.video.set('privacy', videoData.privacy)
 287       options.video.set('channelId', videoData.channelId)
 288       options.video.set('views', videoData.views)
 289
 290       await options.video.save(sequelizeOptions)
 291
 292       if (thumbnailModel) {
 293         thumbnailModel.videoId = options.video.id
 294         options.video.addThumbnail(await thumbnailModel.save({ transaction: t }))
 295       }
 296
 297       // FIXME: use icon URL instead
 298       const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename))
 299       const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 300
 301       options.video.addThumbnail(await previewModel.save({ transaction: t }))
 302
 303       {
 304         const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
 305         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 306
 307         // Remove video files that do not exist anymore
 308         const destroyTasks = options.video.VideoFiles
 309                                     .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
 310                                     .map(f => f.destroy(sequelizeOptions))
 311         await Promise.all(destroyTasks)
 312
 313         // Update or add other one
 314         const upsertTasks = videoFileAttributes.map(a => {
 315           return VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t })
 316             .then(([ file ]) => file)
 317         })
 318
 319         options.video.VideoFiles = await Promise.all(upsertTasks)
 320       }
 321
 322       {
 323         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
 324           options.video,
 325           options.videoObject,
 326           options.video.VideoFiles
 327         )
 328         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 329
 330         // Remove video files that do not exist anymore
 331         const destroyTasks = options.video.VideoStreamingPlaylists
 332                                     .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
 333                                     .map(f => f.destroy(sequelizeOptions))
 334         await Promise.all(destroyTasks)
 335
 336         // Update or add other one
 337         const upsertTasks = streamingPlaylistAttributes.map(a => {
 338           return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
 339                                .then(([ streamingPlaylist ]) => streamingPlaylist)
 340         })
 341
 342         options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
 343       }
 344
 345       {
 346         // Update Tags
 347         const tags = options.videoObject.tag.map(tag => tag.name)
 348         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 349         await options.video.$set('Tags', tagInstances, sequelizeOptions)
 350       }
 351
 352       {
 353         // Update captions
 354         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
 355
 356         const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
 357           return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
 358         })
 359         options.video.VideoCaptions = await Promise.all(videoCaptionsPromises)
 360       }
 361     })
 362
 363     // Notify our users?
 364     if (wasPrivateVideo || wasUnlistedVideo) {
 365       Notifier.Instance.notifyOnNewVideo(options.video)
 366     }
 367
 368     logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
 369   } catch (err) {
 370     if (options.video !== undefined && videoFieldsSave !== undefined) {
 371       resetSequelizeInstance(options.video, videoFieldsSave)
 372     }
 373
 374     // This is just a debug because we will retry the insert
 375     logger.debug('Cannot update the remote video.', { err })
 376     throw err
 377   }
 378 }
 379
 380 async function refreshVideoIfNeeded (options: {
 381   video: VideoModel,
 382   fetchedType: VideoFetchByUrlType,
 383   syncParam: SyncParam
 384 }): Promise<VideoModel> {
 385   if (!options.video.isOutdated()) return options.video
 386
 387   // We need more attributes if the argument video was fetched with not enough joints
 388   const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 389
 390   try {
 391     const { response, videoObject } = await fetchRemoteVideo(video.url)
 392     if (response.statusCode === 404) {
 393       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 394
 395       // Video does not exist anymore
 396       await video.destroy()
 397       return undefined
 398     }
 399
 400     if (videoObject === undefined) {
 401       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 402
 403       await video.setAsRefreshed()
 404       return video
 405     }
 406
 407     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 408     const account = await AccountModel.load(channelActor.VideoChannel.accountId)
 409
 410     const updateOptions = {
 411       video,
 412       videoObject,
 413       account,
 414       channel: channelActor.VideoChannel
 415     }
 416     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 417     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 418
 419     return video
 420   } catch (err) {
 421     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 422
 423     // Don't refresh in loop
 424     await video.setAsRefreshed()
 425     return video
 426   }
 427 }
 428
 429 export {
 430   updateVideoFromAP,
 431   refreshVideoIfNeeded,
 432   federateVideoIfNeeded,
 433   fetchRemoteVideo,
 434   getOrCreateVideoAndAccountAndChannel,
 435   fetchRemoteVideoStaticFile,
 436   fetchRemoteVideoDescription,
 437   getOrCreateVideoChannelFromVideoObject
 438 }
 439
 440 // ---------------------------------------------------------------------------
 441
 442 function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
 443   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 444
 445   const urlMediaType = url.mediaType || url.mimeType
 446   return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
 447 }
 448
 449 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 450   const urlMediaType = url.mediaType || url.mimeType
 451
 452   return urlMediaType === 'application/x-mpegURL'
 453 }
 454
 455 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 456   const urlMediaType = tag.mediaType || tag.mimeType
 457
 458   return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
 459 }
 460
 461 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
 462   logger.debug('Adding remote video %s.', videoObject.id)
 463
 464   const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
 465   const video = VideoModel.build(videoData)
 466
 467   const promiseThumbnail = createVideoThumbnailFromUrl(videoObject.icon.url, video, ThumbnailType.THUMBNAIL)
 468
 469   let thumbnailModel: ThumbnailModel
 470   if (waitThumbnail === true) {
 471     thumbnailModel = await promiseThumbnail
 472   }
 473
 474   const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
 475     const sequelizeOptions = { transaction: t }
 476
 477     const videoCreated = await video.save(sequelizeOptions)
 478     videoCreated.VideoChannel = channelActor.VideoChannel
 479
 480     if (thumbnailModel) {
 481       thumbnailModel.videoId = videoCreated.id
 482
 483       videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t }))
 484     }
 485
 486     // FIXME: use icon URL instead
 487     const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 488     const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 489     previewModel.videoId = videoCreated.id
 490
 491     videoCreated.addThumbnail(await previewModel.save({ transaction: t }))
 492
 493     // Process files
 494     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
 495     if (videoFileAttributes.length === 0) {
 496       throw new Error('Cannot find valid files for video %s ' + videoObject.url)
 497     }
 498
 499     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 500     const videoFiles = await Promise.all(videoFilePromises)
 501
 502     const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 503     const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
 504     await Promise.all(playlistPromises)
 505
 506     // Process tags
 507     const tags = videoObject.tag
 508                             .filter(t => t.type === 'Hashtag')
 509                             .map(t => t.name)
 510     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 511     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 512
 513     // Process captions
 514     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 515       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
 516     })
 517     await Promise.all(videoCaptionsPromises)
 518
 519     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 520
 521     return videoCreated
 522   })
 523
 524   if (waitThumbnail === false) {
 525     promiseThumbnail.then(thumbnailModel => {
 526       thumbnailModel = videoCreated.id
 527
 528       return thumbnailModel.save()
 529     })
 530   }
 531
 532   return videoCreated
 533 }
 534
 535 async function videoActivityObjectToDBAttributes (
 536   videoChannel: VideoChannelModel,
 537   videoObject: VideoTorrentObject,
 538   to: string[] = []
 539 ) {
 540   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
 541   const duration = videoObject.duration.replace(/[^\d]+/, '')
 542
 543   let language: string | undefined
 544   if (videoObject.language) {
 545     language = videoObject.language.identifier
 546   }
 547
 548   let category: number | undefined
 549   if (videoObject.category) {
 550     category = parseInt(videoObject.category.identifier, 10)
 551   }
 552
 553   let licence: number | undefined
 554   if (videoObject.licence) {
 555     licence = parseInt(videoObject.licence.identifier, 10)
 556   }
 557
 558   const description = videoObject.content || null
 559   const support = videoObject.support || null
 560
 561   return {
 562     name: videoObject.name,
 563     uuid: videoObject.uuid,
 564     url: videoObject.id,
 565     category,
 566     licence,
 567     language,
 568     description,
 569     support,
 570     nsfw: videoObject.sensitive,
 571     commentsEnabled: videoObject.commentsEnabled,
 572     downloadEnabled: videoObject.downloadEnabled,
 573     waitTranscoding: videoObject.waitTranscoding,
 574     state: videoObject.state,
 575     channelId: videoChannel.id,
 576     duration: parseInt(duration, 10),
 577     createdAt: new Date(videoObject.published),
 578     publishedAt: new Date(videoObject.published),
 579     originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
 580     // FIXME: updatedAt does not seems to be considered by Sequelize
 581     updatedAt: new Date(videoObject.updated),
 582     views: videoObject.views,
 583     likes: 0,
 584     dislikes: 0,
 585     remote: true,
 586     privacy
 587   }
 588 }
 589
 590 function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
 591   const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 592
 593   if (fileUrls.length === 0) {
 594     throw new Error('Cannot find video files for ' + video.url)
 595   }
 596
 597   const attributes: FilteredModelAttributes<VideoFileModel>[] = []
 598   for (const fileUrl of fileUrls) {
 599     // Fetch associated magnet uri
 600     const magnet = videoObject.url.find(u => {
 601       const mediaType = u.mediaType || u.mimeType
 602       return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
 603     })
 604
 605     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 606
 607     const parsed = magnetUtil.decode(magnet.href)
 608     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 609       throw new Error('Cannot parse magnet URI ' + magnet.href)
 610     }
 611
 612     const mediaType = fileUrl.mediaType || fileUrl.mimeType
 613     const attribute = {
 614       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
 615       infoHash: parsed.infoHash,
 616       resolution: fileUrl.height,
 617       size: fileUrl.size,
 618       videoId: video.id,
 619       fps: fileUrl.fps || -1
 620     }
 621
 622     attributes.push(attribute)
 623   }
 624
 625   return attributes
 626 }
 627
 628 function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
 629   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 630   if (playlistUrls.length === 0) return []
 631
 632   const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
 633   for (const playlistUrlObject of playlistUrls) {
 634     const segmentsSha256UrlObject = playlistUrlObject.tag
 635                                                      .find(t => {
 636                                                        return isAPPlaylistSegmentHashesUrlObject(t)
 637                                                      }) as ActivityPlaylistSegmentHashesObject
 638     if (!segmentsSha256UrlObject) {
 639       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 640       continue
 641     }
 642
 643     const attribute = {
 644       type: VideoStreamingPlaylistType.HLS,
 645       playlistUrl: playlistUrlObject.href,
 646       segmentsSha256Url: segmentsSha256UrlObject.href,
 647       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
 648       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 649       videoId: video.id
 650     }
 651
 652     attributes.push(attribute)
 653   }
 654
 655   return attributes
 656 }