server/lib/activitypub/videos.ts

   1 import * as Bluebird from 'bluebird'
   2 import * as sequelize from 'sequelize'
   3 import * as magnetUtil from 'magnet-uri'
   4 import * as request from 'request'
   5 import {
   6   ActivityPlaylistSegmentHashesObject,
   7   ActivityPlaylistUrlObject,
   8   ActivityUrlObject,
   9   ActivityVideoUrlObject,
  10   VideoState
  11 } from '../../../shared/index'
  12 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
  13 import { VideoPrivacy } from '../../../shared/models/videos'
  14 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
  15 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
  16 import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
  17 import { logger } from '../../helpers/logger'
  18 import { doRequest } from '../../helpers/requests'
  19 import {
  20   ACTIVITY_PUB,
  21   MIMETYPES,
  22   P2P_MEDIA_LOADER_PEER_VERSION,
  23   PREVIEWS_SIZE,
  24   REMOTE_SCHEME,
  25   STATIC_PATHS
  26 } from '../../initializers/constants'
  27 import { ActorModel } from '../../models/activitypub/actor'
  28 import { TagModel } from '../../models/video/tag'
  29 import { VideoModel } from '../../models/video/video'
  30 import { VideoChannelModel } from '../../models/video/video-channel'
  31 import { VideoFileModel } from '../../models/video/video-file'
  32 import { getOrCreateActorAndServerAndModel } from './actor'
  33 import { addVideoComments } from './video-comments'
  34 import { crawlCollectionPage } from './crawl'
  35 import { sendCreateVideo, sendUpdateVideo } from './send'
  36 import { isArray } from '../../helpers/custom-validators/misc'
  37 import { VideoCaptionModel } from '../../models/video/video-caption'
  38 import { JobQueue } from '../job-queue'
  39 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
  40 import { createRates } from './video-rates'
  41 import { addVideoShares, shareVideoByServerAndChannel } from './share'
  42 import { AccountModel } from '../../models/account/account'
  43 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
  44 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
  45 import { Notifier } from '../notifier'
  46 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
  47 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
  48 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
  49 import { VideoShareModel } from '../../models/video/video-share'
  50 import { VideoCommentModel } from '../../models/video/video-comment'
  51 import { sequelizeTypescript } from '../../initializers/database'
  52 import { createPlaceholderThumbnail, createVideoThumbnailFromUrl } from '../thumbnail'
  53 import { ThumbnailModel } from '../../models/video/thumbnail'
  54 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
  55 import { join } from 'path'
  56
  57 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
  58   // If the video is not private and is published, we federate it
  59   if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
  60     // Fetch more attributes that we will need to serialize in AP object
  61     if (isArray(video.VideoCaptions) === false) {
  62       video.VideoCaptions = await video.$get('VideoCaptions', {
  63         attributes: [ 'language' ],
  64         transaction
  65       }) as VideoCaptionModel[]
  66     }
  67
  68     if (isNewVideo) {
  69       // Now we'll add the video's meta data to our followers
  70       await sendCreateVideo(video, transaction)
  71       await shareVideoByServerAndChannel(video, transaction)
  72     } else {
  73       await sendUpdateVideo(video, transaction)
  74     }
  75   }
  76 }
  77
  78 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
  79   const options = {
  80     uri: videoUrl,
  81     method: 'GET',
  82     json: true,
  83     activityPub: true
  84   }
  85
  86   logger.info('Fetching remote video %s.', videoUrl)
  87
  88   const { response, body } = await doRequest(options)
  89
  90   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
  91     logger.debug('Remote video JSON is not valid.', { body })
  92     return { response, videoObject: undefined }
  93   }
  94
  95   return { response, videoObject: body }
  96 }
  97
  98 async function fetchRemoteVideoDescription (video: VideoModel) {
  99   const host = video.VideoChannel.Account.Actor.Server.host
 100   const path = video.getDescriptionAPIPath()
 101   const options = {
 102     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
 103     json: true
 104   }
 105
 106   const { body } = await doRequest(options)
 107   return body.description ? body.description : ''
 108 }
 109
 110 function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
 111   const url = buildRemoteBaseUrl(video, path)
 112
 113   // We need to provide a callback, if no we could have an uncaught exception
 114   return request.get(url, err => {
 115     if (err) reject(err)
 116   })
 117 }
 118
 119 function buildRemoteBaseUrl (video: VideoModel, path: string) {
 120   const host = video.VideoChannel.Account.Actor.Server.host
 121
 122   return REMOTE_SCHEME.HTTP + '://' + host + path
 123 }
 124
 125 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
 126   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
 127   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
 128
 129   if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
 130     throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
 131   }
 132
 133   return getOrCreateActorAndServerAndModel(channel.id, 'all')
 134 }
 135
 136 type SyncParam = {
 137   likes: boolean
 138   dislikes: boolean
 139   shares: boolean
 140   comments: boolean
 141   thumbnail: boolean
 142   refreshVideo?: boolean
 143 }
 144 async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
 145   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 146
 147   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 148
 149   if (syncParam.likes === true) {
 150     const handler = items => createRates(items, video, 'like')
 151     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
 152
 153     await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
 154       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
 155   } else {
 156     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
 157   }
 158
 159   if (syncParam.dislikes === true) {
 160     const handler = items => createRates(items, video, 'dislike')
 161     const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
 162
 163     await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
 164       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
 165   } else {
 166     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
 167   }
 168
 169   if (syncParam.shares === true) {
 170     const handler = items => addVideoShares(items, video)
 171     const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
 172
 173     await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
 174       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
 175   } else {
 176     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
 177   }
 178
 179   if (syncParam.comments === true) {
 180     const handler = items => addVideoComments(items, video)
 181     const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
 182
 183     await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
 184       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
 185   } else {
 186     jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
 187   }
 188
 189   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
 190 }
 191
 192 async function getOrCreateVideoAndAccountAndChannel (options: {
 193   videoObject: { id: string } | string,
 194   syncParam?: SyncParam,
 195   fetchType?: VideoFetchByUrlType,
 196   allowRefresh?: boolean // true by default
 197 }) {
 198   // Default params
 199   const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 200   const fetchType = options.fetchType || 'all'
 201   const allowRefresh = options.allowRefresh !== false
 202
 203   // Get video url
 204   const videoUrl = getAPId(options.videoObject)
 205
 206   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
 207   if (videoFromDatabase) {
 208     if (videoFromDatabase.isOutdated() && allowRefresh === true) {
 209       const refreshOptions = {
 210         video: videoFromDatabase,
 211         fetchedType: fetchType,
 212         syncParam
 213       }
 214
 215       if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
 216       else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
 217     }
 218
 219     return { video: videoFromDatabase, created: false }
 220   }
 221
 222   const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
 223   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 224
 225   const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
 226   const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
 227
 228   await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
 229
 230   return { video, created: true }
 231 }
 232
 233 async function updateVideoFromAP (options: {
 234   video: VideoModel,
 235   videoObject: VideoTorrentObject,
 236   account: AccountModel,
 237   channel: VideoChannelModel,
 238   overrideTo?: string[]
 239 }) {
 240   logger.debug('Updating remote video "%s".', options.videoObject.uuid)
 241
 242   let videoFieldsSave: any
 243   const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE
 244   const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
 245
 246   try {
 247     let thumbnailModel: ThumbnailModel
 248
 249     try {
 250       thumbnailModel = await createVideoThumbnailFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.THUMBNAIL)
 251     } catch (err) {
 252       logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
 253     }
 254
 255     await sequelizeTypescript.transaction(async t => {
 256       const sequelizeOptions = { transaction: t }
 257
 258       videoFieldsSave = options.video.toJSON()
 259
 260       // Check actor has the right to update the video
 261       const videoChannel = options.video.VideoChannel
 262       if (videoChannel.Account.id !== options.account.id) {
 263         throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
 264       }
 265
 266       const to = options.overrideTo ? options.overrideTo : options.videoObject.to
 267       const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
 268       options.video.set('name', videoData.name)
 269       options.video.set('uuid', videoData.uuid)
 270       options.video.set('url', videoData.url)
 271       options.video.set('category', videoData.category)
 272       options.video.set('licence', videoData.licence)
 273       options.video.set('language', videoData.language)
 274       options.video.set('description', videoData.description)
 275       options.video.set('support', videoData.support)
 276       options.video.set('nsfw', videoData.nsfw)
 277       options.video.set('commentsEnabled', videoData.commentsEnabled)
 278       options.video.set('downloadEnabled', videoData.downloadEnabled)
 279       options.video.set('waitTranscoding', videoData.waitTranscoding)
 280       options.video.set('state', videoData.state)
 281       options.video.set('duration', videoData.duration)
 282       options.video.set('createdAt', videoData.createdAt)
 283       options.video.set('publishedAt', videoData.publishedAt)
 284       options.video.set('originallyPublishedAt', videoData.originallyPublishedAt)
 285       options.video.set('privacy', videoData.privacy)
 286       options.video.set('channelId', videoData.channelId)
 287       options.video.set('views', videoData.views)
 288
 289       await options.video.save(sequelizeOptions)
 290
 291       if (thumbnailModel) {
 292         thumbnailModel.videoId = options.video.id
 293         options.video.addThumbnail(await thumbnailModel.save({ transaction: t }))
 294       }
 295
 296       // FIXME: use icon URL instead
 297       const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename))
 298       const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 299
 300       options.video.addThumbnail(await previewModel.save({ transaction: t }))
 301
 302       {
 303         const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
 304         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 305
 306         // Remove video files that do not exist anymore
 307         const destroyTasks = options.video.VideoFiles
 308                                     .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
 309                                     .map(f => f.destroy(sequelizeOptions))
 310         await Promise.all(destroyTasks)
 311
 312         // Update or add other one
 313         const upsertTasks = videoFileAttributes.map(a => {
 314           return (VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t }) as any) // FIXME: sequelize typings
 315             .then(([ file ]) => file)
 316         })
 317
 318         options.video.VideoFiles = await Promise.all(upsertTasks)
 319       }
 320
 321       {
 322         const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
 323           options.video,
 324           options.videoObject,
 325           options.video.VideoFiles
 326         )
 327         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 328
 329         // Remove video files that do not exist anymore
 330         const destroyTasks = options.video.VideoStreamingPlaylists
 331                                     .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
 332                                     .map(f => f.destroy(sequelizeOptions))
 333         await Promise.all(destroyTasks)
 334
 335         // Update or add other one
 336         const upsertTasks = streamingPlaylistAttributes.map(a => {
 337           // FIXME: sequelize typings
 338           return (VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) as any)
 339                                .then(([ streamingPlaylist ]) => streamingPlaylist)
 340         })
 341
 342         options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
 343       }
 344
 345       {
 346         // Update Tags
 347         const tags = options.videoObject.tag.map(tag => tag.name)
 348         const tagInstances = await TagModel.findOrCreateTags(tags, t)
 349         await options.video.$set('Tags', tagInstances, sequelizeOptions)
 350       }
 351
 352       {
 353         // Update captions
 354         await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
 355
 356         const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
 357           return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
 358         })
 359         options.video.VideoCaptions = await Promise.all(videoCaptionsPromises)
 360       }
 361     })
 362
 363     // Notify our users?
 364     if (wasPrivateVideo || wasUnlistedVideo) {
 365       Notifier.Instance.notifyOnNewVideo(options.video)
 366     }
 367
 368     logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
 369   } catch (err) {
 370     if (options.video !== undefined && videoFieldsSave !== undefined) {
 371       resetSequelizeInstance(options.video, videoFieldsSave)
 372     }
 373
 374     // This is just a debug because we will retry the insert
 375     logger.debug('Cannot update the remote video.', { err })
 376     throw err
 377   }
 378 }
 379
 380 async function refreshVideoIfNeeded (options: {
 381   video: VideoModel,
 382   fetchedType: VideoFetchByUrlType,
 383   syncParam: SyncParam
 384 }): Promise<VideoModel> {
 385   if (!options.video.isOutdated()) return options.video
 386
 387   // We need more attributes if the argument video was fetched with not enough joints
 388   const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 389
 390   try {
 391     const { response, videoObject } = await fetchRemoteVideo(video.url)
 392     if (response.statusCode === 404) {
 393       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 394
 395       // Video does not exist anymore
 396       await video.destroy()
 397       return undefined
 398     }
 399
 400     if (videoObject === undefined) {
 401       logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
 402
 403       await video.setAsRefreshed()
 404       return video
 405     }
 406
 407     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 408     const account = await AccountModel.load(channelActor.VideoChannel.accountId)
 409
 410     const updateOptions = {
 411       video,
 412       videoObject,
 413       account,
 414       channel: channelActor.VideoChannel
 415     }
 416     await retryTransactionWrapper(updateVideoFromAP, updateOptions)
 417     await syncVideoExternalAttributes(video, videoObject, options.syncParam)
 418
 419     return video
 420   } catch (err) {
 421     logger.warn('Cannot refresh video %s.', options.video.url, { err })
 422
 423     // Don't refresh in loop
 424     await video.setAsRefreshed()
 425     return video
 426   }
 427 }
 428
 429 export {
 430   updateVideoFromAP,
 431   refreshVideoIfNeeded,
 432   federateVideoIfNeeded,
 433   fetchRemoteVideo,
 434   getOrCreateVideoAndAccountAndChannel,
 435   fetchRemoteVideoStaticFile,
 436   fetchRemoteVideoDescription,
 437   getOrCreateVideoChannelFromVideoObject
 438 }
 439
 440 // ---------------------------------------------------------------------------
 441
 442 function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
 443   const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
 444
 445   const urlMediaType = url.mediaType || url.mimeType
 446   return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
 447 }
 448
 449 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
 450   const urlMediaType = url.mediaType || url.mimeType
 451
 452   return urlMediaType === 'application/x-mpegURL'
 453 }
 454
 455 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
 456   const urlMediaType = tag.mediaType || tag.mimeType
 457
 458   return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
 459 }
 460
 461 async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
 462   logger.debug('Adding remote video %s.', videoObject.id)
 463
 464   const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
 465   const video = VideoModel.build(videoData)
 466
 467   const promiseThumbnail = createVideoThumbnailFromUrl(videoObject.icon.url, video, ThumbnailType.THUMBNAIL)
 468
 469   let thumbnailModel: ThumbnailModel
 470   if (waitThumbnail === true) {
 471     thumbnailModel = await promiseThumbnail
 472   }
 473
 474   const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
 475     const sequelizeOptions = { transaction: t }
 476
 477     const videoCreated = await video.save(sequelizeOptions)
 478     videoCreated.VideoChannel = channelActor.VideoChannel
 479
 480     if (thumbnailModel) {
 481       thumbnailModel.videoId = videoCreated.id
 482
 483       videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t }))
 484     }
 485
 486     // FIXME: use icon URL instead
 487     const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
 488     const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
 489     previewModel.videoId = videoCreated.id
 490
 491     videoCreated.addThumbnail(await previewModel.save({ transaction: t }))
 492
 493     // Process files
 494     const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
 495     if (videoFileAttributes.length === 0) {
 496       throw new Error('Cannot find valid files for video %s ' + videoObject.url)
 497     }
 498
 499     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
 500     const videoFiles = await Promise.all(videoFilePromises)
 501
 502     const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
 503     const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
 504     await Promise.all(playlistPromises)
 505
 506     // Process tags
 507     const tags = videoObject.tag
 508                             .filter(t => t.type === 'Hashtag')
 509                             .map(t => t.name)
 510     const tagInstances = await TagModel.findOrCreateTags(tags, t)
 511     await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
 512
 513     // Process captions
 514     const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
 515       return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
 516     })
 517     await Promise.all(videoCaptionsPromises)
 518
 519     logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
 520
 521     return videoCreated
 522   })
 523
 524   if (waitThumbnail === false) {
 525     promiseThumbnail.then(thumbnailModel => {
 526       thumbnailModel = videoCreated.id
 527
 528       return thumbnailModel.save()
 529     })
 530   }
 531
 532   return videoCreated
 533 }
 534
 535 async function videoActivityObjectToDBAttributes (
 536   videoChannel: VideoChannelModel,
 537   videoObject: VideoTorrentObject,
 538   to: string[] = []
 539 ) {
 540   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
 541   const duration = videoObject.duration.replace(/[^\d]+/, '')
 542
 543   let language: string | undefined
 544   if (videoObject.language) {
 545     language = videoObject.language.identifier
 546   }
 547
 548   let category: number | undefined
 549   if (videoObject.category) {
 550     category = parseInt(videoObject.category.identifier, 10)
 551   }
 552
 553   let licence: number | undefined
 554   if (videoObject.licence) {
 555     licence = parseInt(videoObject.licence.identifier, 10)
 556   }
 557
 558   const description = videoObject.content || null
 559   const support = videoObject.support || null
 560
 561   return {
 562     name: videoObject.name,
 563     uuid: videoObject.uuid,
 564     url: videoObject.id,
 565     category,
 566     licence,
 567     language,
 568     description,
 569     support,
 570     nsfw: videoObject.sensitive,
 571     commentsEnabled: videoObject.commentsEnabled,
 572     downloadEnabled: videoObject.downloadEnabled,
 573     waitTranscoding: videoObject.waitTranscoding,
 574     state: videoObject.state,
 575     channelId: videoChannel.id,
 576     duration: parseInt(duration, 10),
 577     createdAt: new Date(videoObject.published),
 578     publishedAt: new Date(videoObject.published),
 579     originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
 580     // FIXME: updatedAt does not seems to be considered by Sequelize
 581     updatedAt: new Date(videoObject.updated),
 582     views: videoObject.views,
 583     likes: 0,
 584     dislikes: 0,
 585     remote: true,
 586     privacy
 587   }
 588 }
 589
 590 function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
 591   const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
 592
 593   if (fileUrls.length === 0) {
 594     throw new Error('Cannot find video files for ' + video.url)
 595   }
 596
 597   const attributes: object[] = [] // FIXME: add typings
 598   for (const fileUrl of fileUrls) {
 599     // Fetch associated magnet uri
 600     const magnet = videoObject.url.find(u => {
 601       const mediaType = u.mediaType || u.mimeType
 602       return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
 603     })
 604
 605     if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
 606
 607     const parsed = magnetUtil.decode(magnet.href)
 608     if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
 609       throw new Error('Cannot parse magnet URI ' + magnet.href)
 610     }
 611
 612     const mediaType = fileUrl.mediaType || fileUrl.mimeType
 613     const attribute = {
 614       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
 615       infoHash: parsed.infoHash,
 616       resolution: fileUrl.height,
 617       size: fileUrl.size,
 618       videoId: video.id,
 619       fps: fileUrl.fps || -1
 620     }
 621
 622     attributes.push(attribute)
 623   }
 624
 625   return attributes
 626 }
 627
 628 function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
 629   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
 630   if (playlistUrls.length === 0) return []
 631
 632   const attributes: object[] = [] // FIXME: add typings
 633   for (const playlistUrlObject of playlistUrls) {
 634     const segmentsSha256UrlObject = playlistUrlObject.tag
 635                                                      .find(t => {
 636                                                        return isAPPlaylistSegmentHashesUrlObject(t)
 637                                                      }) as ActivityPlaylistSegmentHashesObject
 638     if (!segmentsSha256UrlObject) {
 639       logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
 640       continue
 641     }
 642
 643     const attribute = {
 644       type: VideoStreamingPlaylistType.HLS,
 645       playlistUrl: playlistUrlObject.href,
 646       segmentsSha256Url: segmentsSha256UrlObject.href,
 647       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
 648       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 649       videoId: video.id
 650     }
 651
 652     attributes.push(attribute)
 653   }
 654
 655   return attributes
 656 }