server/lib/activitypub/playlists/create-update.ts

   1 import { isArray } from '@server/helpers/custom-validators/misc'
   2 import { logger, loggerTagsFactory } from '@server/helpers/logger'
   3 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
   4 import { sequelizeTypescript } from '@server/initializers/database'
   5 import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
   6 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
   7 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
   8 import { FilteredModelAttributes } from '@server/types'
   9 import { MAccountDefault, MAccountId, MThumbnail, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
  10 import { AttributesOnly } from '@shared/core-utils'
  11 import { PlaylistObject } from '@shared/models'
  12 import { getOrCreateAPActor } from '../actors'
  13 import { crawlCollectionPage } from '../crawl'
  14 import { getOrCreateAPVideo } from '../videos'
  15 import {
  16   fetchRemotePlaylistElement,
  17   fetchRemoteVideoPlaylist,
  18   playlistElementObjectToDBAttributes,
  19   playlistObjectToDBAttributes
  20 } from './shared'
  21
  22 import Bluebird = require('bluebird')
  23
  24 const lTags = loggerTagsFactory('ap', 'video-playlist')
  25
  26 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  27   await Bluebird.map(playlistUrls, async playlistUrl => {
  28     try {
  29       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  30       if (exists === true) return
  31
  32       const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
  33
  34       if (playlistObject === undefined) {
  35         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
  36       }
  37
  38       return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
  39     } catch (err) {
  40       logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
  41     }
  42   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  43 }
  44
  45 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  46   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  47
  48   await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
  49
  50   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  51
  52   const playlistElementUrls = await fetchElementUrls(playlistObject)
  53
  54   // Refetch playlist from DB since elements fetching could be long in time
  55   const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
  56
  57   await updatePlaylistThumbnail(playlistObject, playlist)
  58
  59   return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
  60 }
  61
  62 // ---------------------------------------------------------------------------
  63
  64 export {
  65   createAccountPlaylists,
  66   createOrUpdateVideoPlaylist
  67 }
  68
  69 // ---------------------------------------------------------------------------
  70
  71 async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
  72   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
  73
  74   const actor = await getOrCreateAPActor(playlistObject.attributedTo[0])
  75
  76   if (!actor.VideoChannel) {
  77     logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
  78     return
  79   }
  80
  81   playlistAttributes.videoChannelId = actor.VideoChannel.id
  82 }
  83
  84 async function fetchElementUrls (playlistObject: PlaylistObject) {
  85   let accItems: string[] = []
  86   await crawlCollectionPage<string>(playlistObject.id, items => {
  87     accItems = accItems.concat(items)
  88
  89     return Promise.resolve()
  90   })
  91
  92   return accItems
  93 }
  94
  95 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
  96   if (playlistObject.icon) {
  97     let thumbnailModel: MThumbnail
  98
  99     try {
 100       thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
 101       await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
 102     } catch (err) {
 103       logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
 104
 105       if (thumbnailModel) await thumbnailModel.removeThumbnail()
 106     }
 107
 108     return
 109   }
 110
 111   // Playlist does not have an icon, destroy existing one
 112   if (playlist.hasThumbnail()) {
 113     await playlist.Thumbnail.destroy()
 114     playlist.Thumbnail = null
 115   }
 116 }
 117
 118 async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 119   const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
 120
 121   await sequelizeTypescript.transaction(async t => {
 122     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 123
 124     for (const element of elementsToCreate) {
 125       await VideoPlaylistElementModel.create(element, { transaction: t })
 126     }
 127   })
 128
 129   logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
 130
 131   return undefined
 132 }
 133
 134 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
 135   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 136
 137   await Bluebird.map(elementUrls, async elementUrl => {
 138     try {
 139       const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
 140
 141       const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
 142
 143       elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
 144     } catch (err) {
 145       logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
 146     }
 147   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 148
 149   return elementsToCreate
 150 }