server/lib/activitypub/playlists/create-update.ts

   1 import { isArray } from '@server/helpers/custom-validators/misc'
   2 import { logger, loggerTagsFactory } from '@server/helpers/logger'
   3 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
   4 import { sequelizeTypescript } from '@server/initializers/database'
   5 import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
   6 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
   7 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
   8 import { FilteredModelAttributes } from '@server/types'
   9 import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
  10 import { AttributesOnly } from '@shared/core-utils'
  11 import { PlaylistObject } from '@shared/models'
  12 import { getOrCreateAPActor } from '../actors'
  13 import { crawlCollectionPage } from '../crawl'
  14 import { getOrCreateAPVideo } from '../videos'
  15 import {
  16   fetchRemotePlaylistElement,
  17   fetchRemoteVideoPlaylist,
  18   playlistElementObjectToDBAttributes,
  19   playlistObjectToDBAttributes
  20 } from './shared'
  21
  22 import Bluebird = require('bluebird')
  23
  24 const lTags = loggerTagsFactory('ap', 'video-playlist')
  25
  26 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  27   await Bluebird.map(playlistUrls, async playlistUrl => {
  28     try {
  29       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  30       if (exists === true) return
  31
  32       const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
  33
  34       if (playlistObject === undefined) {
  35         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
  36       }
  37
  38       return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
  39     } catch (err) {
  40       logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
  41     }
  42   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  43 }
  44
  45 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  46   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  47
  48   await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
  49
  50   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  51
  52   const playlistElementUrls = await fetchElementUrls(playlistObject)
  53
  54   // Refetch playlist from DB since elements fetching could be long in time
  55   const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
  56
  57   try {
  58     await updatePlaylistThumbnail(playlistObject, playlist)
  59   } catch (err) {
  60     logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
  61   }
  62
  63   return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
  64 }
  65
  66 // ---------------------------------------------------------------------------
  67
  68 export {
  69   createAccountPlaylists,
  70   createOrUpdateVideoPlaylist
  71 }
  72
  73 // ---------------------------------------------------------------------------
  74
  75 async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
  76   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
  77
  78   const actor = await getOrCreateAPActor(playlistObject.attributedTo[0])
  79
  80   if (!actor.VideoChannel) {
  81     logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
  82     return
  83   }
  84
  85   playlistAttributes.videoChannelId = actor.VideoChannel.id
  86 }
  87
  88 async function fetchElementUrls (playlistObject: PlaylistObject) {
  89   let accItems: string[] = []
  90   await crawlCollectionPage<string>(playlistObject.id, items => {
  91     accItems = accItems.concat(items)
  92
  93     return Promise.resolve()
  94   })
  95
  96   return accItems
  97 }
  98
  99 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
 100   if (playlistObject.icon) {
 101     const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
 102     await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
 103
 104     return
 105   }
 106
 107   // Playlist does not have an icon, destroy existing one
 108   if (playlist.hasThumbnail()) {
 109     await playlist.Thumbnail.destroy()
 110     playlist.Thumbnail = null
 111   }
 112 }
 113
 114 async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 115   const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
 116
 117   await sequelizeTypescript.transaction(async t => {
 118     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 119
 120     for (const element of elementsToCreate) {
 121       await VideoPlaylistElementModel.create(element, { transaction: t })
 122     }
 123   })
 124
 125   logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
 126
 127   return undefined
 128 }
 129
 130 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
 131   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 132
 133   await Bluebird.map(elementUrls, async elementUrl => {
 134     try {
 135       const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
 136
 137       const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
 138
 139       elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
 140     } catch (err) {
 141       logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
 142     }
 143   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 144
 145   return elementsToCreate
 146 }