server/lib/activitypub/playlists/create-update.ts

   1 import { map } from 'bluebird'
   2 import { isArray } from '@server/helpers/custom-validators/misc'
   3 import { retryTransactionWrapper } from '@server/helpers/database-utils'
   4 import { logger, loggerTagsFactory } from '@server/helpers/logger'
   5 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
   6 import { sequelizeTypescript } from '@server/initializers/database'
   7 import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
   8 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
   9 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
  10 import { FilteredModelAttributes } from '@server/types'
  11 import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
  12 import { PlaylistObject } from '@shared/models'
  13 import { AttributesOnly } from '@shared/typescript-utils'
  14 import { getAPId } from '../activity'
  15 import { getOrCreateAPActor } from '../actors'
  16 import { crawlCollectionPage } from '../crawl'
  17 import { getOrCreateAPVideo } from '../videos'
  18 import {
  19   fetchRemotePlaylistElement,
  20   fetchRemoteVideoPlaylist,
  21   playlistElementObjectToDBAttributes,
  22   playlistObjectToDBAttributes
  23 } from './shared'
  24
  25 const lTags = loggerTagsFactory('ap', 'video-playlist')
  26
  27 async function createAccountPlaylists (playlistUrls: string[]) {
  28   await map(playlistUrls, async playlistUrl => {
  29     try {
  30       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  31       if (exists === true) return
  32
  33       const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
  34
  35       if (playlistObject === undefined) {
  36         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
  37       }
  38
  39       return createOrUpdateVideoPlaylist(playlistObject)
  40     } catch (err) {
  41       logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
  42     }
  43   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  44 }
  45
  46 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
  47   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
  48
  49   await setVideoChannel(playlistObject, playlistAttributes)
  50
  51   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
  52
  53   const playlistElementUrls = await fetchElementUrls(playlistObject)
  54
  55   // Refetch playlist from DB since elements fetching could be long in time
  56   const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
  57
  58   await updatePlaylistThumbnail(playlistObject, playlist)
  59
  60   const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
  61   playlist.setVideosLength(elementsLength)
  62
  63   return playlist
  64 }
  65
  66 // ---------------------------------------------------------------------------
  67
  68 export {
  69   createAccountPlaylists,
  70   createOrUpdateVideoPlaylist
  71 }
  72
  73 // ---------------------------------------------------------------------------
  74
  75 async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
  76   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
  77     throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
  78   }
  79
  80   const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
  81
  82   if (!actor.VideoChannel) {
  83     logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
  84     return
  85   }
  86
  87   playlistAttributes.videoChannelId = actor.VideoChannel.id
  88   playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
  89 }
  90
  91 async function fetchElementUrls (playlistObject: PlaylistObject) {
  92   let accItems: string[] = []
  93   await crawlCollectionPage<string>(playlistObject.id, items => {
  94     accItems = accItems.concat(items)
  95
  96     return Promise.resolve()
  97   })
  98
  99   return accItems
 100 }
 101
 102 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
 103   if (playlistObject.icon) {
 104     let thumbnailModel: MThumbnail
 105
 106     try {
 107       thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
 108       await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
 109     } catch (err) {
 110       logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
 111
 112       if (thumbnailModel) await thumbnailModel.removeThumbnail()
 113     }
 114
 115     return
 116   }
 117
 118   // Playlist does not have an icon, destroy existing one
 119   if (playlist.hasThumbnail()) {
 120     await playlist.Thumbnail.destroy()
 121     playlist.Thumbnail = null
 122   }
 123 }
 124
 125 async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 126   const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
 127
 128   await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => {
 129     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 130
 131     for (const element of elementsToCreate) {
 132       await VideoPlaylistElementModel.create(element, { transaction: t })
 133     }
 134   }))
 135
 136   logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
 137
 138   return elementsToCreate.length
 139 }
 140
 141 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
 142   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 143
 144   await map(elementUrls, async elementUrl => {
 145     try {
 146       const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
 147
 148       const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
 149
 150       elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
 151     } catch (err) {
 152       logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
 153     }
 154   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 155
 156   return elementsToCreate
 157 }