server/lib/activitypub/playlist.ts

   1 import * as Bluebird from 'bluebird'
   2 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
   3 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
   4 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   5 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
   6 import { checkUrlsSameHost } from '../../helpers/activitypub'
   7 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
   8 import { isArray } from '../../helpers/custom-validators/misc'
   9 import { logger } from '../../helpers/logger'
  10 import { doJSONRequest } from '../../helpers/requests'
  11 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
  12 import { sequelizeTypescript } from '../../initializers/database'
  13 import { VideoPlaylistModel } from '../../models/video/video-playlist'
  14 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  15 import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
  16 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
  17 import { FilteredModelAttributes } from '../../types/sequelize'
  18 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  19 import { getOrCreateActorAndServerAndModel } from './actor'
  20 import { crawlCollectionPage } from './crawl'
  21 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  22
  23 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  24   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
  25     ? VideoPlaylistPrivacy.PUBLIC
  26     : VideoPlaylistPrivacy.UNLISTED
  27
  28   return {
  29     name: playlistObject.name,
  30     description: playlistObject.content,
  31     privacy,
  32     url: playlistObject.id,
  33     uuid: playlistObject.uuid,
  34     ownerAccountId: byAccount.id,
  35     videoChannelId: null,
  36     createdAt: new Date(playlistObject.published),
  37     updatedAt: new Date(playlistObject.updated)
  38   }
  39 }
  40
  41 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
  42   return {
  43     position: elementObject.position,
  44     url: elementObject.id,
  45     startTimestamp: elementObject.startTimestamp || null,
  46     stopTimestamp: elementObject.stopTimestamp || null,
  47     videoPlaylistId: videoPlaylist.id,
  48     videoId: video.id
  49   }
  50 }
  51
  52 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  53   await Bluebird.map(playlistUrls, async playlistUrl => {
  54     try {
  55       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  56       if (exists === true) return
  57
  58       // Fetch url
  59       const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
  60
  61       if (!isPlaylistObjectValid(body)) {
  62         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  63       }
  64
  65       if (!isArray(body.to)) {
  66         throw new Error('Playlist does not have an audience.')
  67       }
  68
  69       return createOrUpdateVideoPlaylist(body, account, body.to)
  70     } catch (err) {
  71       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  72     }
  73   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  74 }
  75
  76 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  77   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  78
  79   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  80     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  81
  82     if (actor.VideoChannel) {
  83       playlistAttributes.videoChannelId = actor.VideoChannel.id
  84     } else {
  85       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  86     }
  87   }
  88
  89   const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  90
  91   let accItems: string[] = []
  92   await crawlCollectionPage<string>(playlistObject.id, items => {
  93     accItems = accItems.concat(items)
  94
  95     return Promise.resolve()
  96   })
  97
  98   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
  99
 100   if (playlistObject.icon) {
 101     try {
 102       const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
 103       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 104     } catch (err) {
 105       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 106     }
 107   } else if (refreshedPlaylist.hasThumbnail()) {
 108     await refreshedPlaylist.Thumbnail.destroy()
 109     refreshedPlaylist.Thumbnail = null
 110   }
 111
 112   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 113 }
 114
 115 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
 116   if (!videoPlaylist.isOutdated()) return videoPlaylist
 117
 118   try {
 119     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 120     if (statusCode === HttpStatusCode.NOT_FOUND_404) {
 121       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 122
 123       await videoPlaylist.destroy()
 124       return undefined
 125     }
 126
 127     if (playlistObject === undefined) {
 128       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 129
 130       await videoPlaylist.setAsRefreshed()
 131       return videoPlaylist
 132     }
 133
 134     const byAccount = videoPlaylist.OwnerAccount
 135     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 136
 137     return videoPlaylist
 138   } catch (err) {
 139     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 140
 141     await videoPlaylist.setAsRefreshed()
 142     return videoPlaylist
 143   }
 144 }
 145
 146 // ---------------------------------------------------------------------------
 147
 148 export {
 149   createAccountPlaylists,
 150   playlistObjectToDBAttributes,
 151   playlistElementObjectToDBAttributes,
 152   createOrUpdateVideoPlaylist,
 153   refreshVideoPlaylistIfNeeded
 154 }
 155
 156 // ---------------------------------------------------------------------------
 157
 158 async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 159   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 160
 161   await Bluebird.map(elementUrls, async elementUrl => {
 162     try {
 163       const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
 164
 165       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 166
 167       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 168         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 169       }
 170
 171       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 172
 173       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 174     } catch (err) {
 175       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 176     }
 177   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 178
 179   await sequelizeTypescript.transaction(async t => {
 180     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 181
 182     for (const element of elementsToCreate) {
 183       await VideoPlaylistElementModel.create(element, { transaction: t })
 184     }
 185   })
 186
 187   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 188
 189   return undefined
 190 }
 191
 192 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 193   logger.info('Fetching remote playlist %s.', playlistUrl)
 194
 195   const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
 196
 197   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 198     logger.debug('Remote video playlist JSON is not valid.', { body })
 199     return { statusCode, playlistObject: undefined }
 200   }
 201
 202   return { statusCode, playlistObject: body }
 203 }