server/lib/activitypub/playlists/create-update.ts

   1 import { map } from 'bluebird'
   2 import { getAPId } from '@server/helpers/activitypub'
   3 import { isArray } from '@server/helpers/custom-validators/misc'
   4 import { logger, loggerTagsFactory } from '@server/helpers/logger'
   5 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
   6 import { sequelizeTypescript } from '@server/initializers/database'
   7 import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
   8 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
   9 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
  10 import { FilteredModelAttributes } from '@server/types'
  11 import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
  12 import { AttributesOnly } from '@shared/typescript-utils'
  13 import { PlaylistObject } from '@shared/models'
  14 import { getOrCreateAPActor } from '../actors'
  15 import { crawlCollectionPage } from '../crawl'
  16 import { getOrCreateAPVideo } from '../videos'
  17 import {
  18   fetchRemotePlaylistElement,
  19   fetchRemoteVideoPlaylist,
  20   playlistElementObjectToDBAttributes,
  21   playlistObjectToDBAttributes
  22 } from './shared'
  23
  24 const lTags = loggerTagsFactory('ap', 'video-playlist')
  25
  26 async function createAccountPlaylists (playlistUrls: string[]) {
  27   await map(playlistUrls, async playlistUrl => {
  28     try {
  29       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  30       if (exists === true) return
  31
  32       const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
  33
  34       if (playlistObject === undefined) {
  35         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
  36       }
  37
  38       return createOrUpdateVideoPlaylist(playlistObject)
  39     } catch (err) {
  40       logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
  41     }
  42   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  43 }
  44
  45 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
  46   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
  47
  48   await setVideoChannel(playlistObject, playlistAttributes)
  49
  50   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
  51
  52   const playlistElementUrls = await fetchElementUrls(playlistObject)
  53
  54   // Refetch playlist from DB since elements fetching could be long in time
  55   const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
  56
  57   await updatePlaylistThumbnail(playlistObject, playlist)
  58
  59   const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
  60   playlist.setVideosLength(elementsLength)
  61
  62   return playlist
  63 }
  64
  65 // ---------------------------------------------------------------------------
  66
  67 export {
  68   createAccountPlaylists,
  69   createOrUpdateVideoPlaylist
  70 }
  71
  72 // ---------------------------------------------------------------------------
  73
  74 async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
  75   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
  76     throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
  77   }
  78
  79   const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
  80
  81   if (!actor.VideoChannel) {
  82     logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
  83     return
  84   }
  85
  86   playlistAttributes.videoChannelId = actor.VideoChannel.id
  87   playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
  88 }
  89
  90 async function fetchElementUrls (playlistObject: PlaylistObject) {
  91   let accItems: string[] = []
  92   await crawlCollectionPage<string>(playlistObject.id, items => {
  93     accItems = accItems.concat(items)
  94
  95     return Promise.resolve()
  96   })
  97
  98   return accItems
  99 }
 100
 101 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
 102   if (playlistObject.icon) {
 103     let thumbnailModel: MThumbnail
 104
 105     try {
 106       thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
 107       await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
 108     } catch (err) {
 109       logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
 110
 111       if (thumbnailModel) await thumbnailModel.removeThumbnail()
 112     }
 113
 114     return
 115   }
 116
 117   // Playlist does not have an icon, destroy existing one
 118   if (playlist.hasThumbnail()) {
 119     await playlist.Thumbnail.destroy()
 120     playlist.Thumbnail = null
 121   }
 122 }
 123
 124 async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 125   const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
 126
 127   await sequelizeTypescript.transaction(async t => {
 128     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 129
 130     for (const element of elementsToCreate) {
 131       await VideoPlaylistElementModel.create(element, { transaction: t })
 132     }
 133   })
 134
 135   logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
 136
 137   return elementsToCreate.length
 138 }
 139
 140 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
 141   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 142
 143   await map(elementUrls, async elementUrl => {
 144     try {
 145       const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
 146
 147       const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
 148
 149       elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
 150     } catch (err) {
 151       logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
 152     }
 153   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 154
 155   return elementsToCreate
 156 }