server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
   4 import { AccountModel } from '../../models/account/account'
   5 import { isArray } from '../../helpers/custom-validators/misc'
   6 import { getOrCreateActorAndServerAndModel } from './actor'
   7 import { logger } from '../../helpers/logger'
   8 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   9 import { doRequest } from '../../helpers/requests'
  10 import { checkUrlsSameHost } from '../../helpers/activitypub'
  11 import * as Bluebird from 'bluebird'
  12 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  13 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  14 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  15 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  16 import { VideoModel } from '../../models/video/video'
  17 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  18 import { sequelizeTypescript } from '../../initializers/database'
  19 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  20 import { FilteredModelAttributes } from '../../typings/sequelize'
  21 import { AccountModelId } from '../../typings/models'
  22
  23 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModelId, to: string[]) {
  24   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
  25
  26   return {
  27     name: playlistObject.name,
  28     description: playlistObject.content,
  29     privacy,
  30     url: playlistObject.id,
  31     uuid: playlistObject.uuid,
  32     ownerAccountId: byAccount.id,
  33     videoChannelId: null,
  34     createdAt: new Date(playlistObject.published),
  35     updatedAt: new Date(playlistObject.updated)
  36   }
  37 }
  38
  39 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) {
  40   return {
  41     position: elementObject.position,
  42     url: elementObject.id,
  43     startTimestamp: elementObject.startTimestamp || null,
  44     stopTimestamp: elementObject.stopTimestamp || null,
  45     videoPlaylistId: videoPlaylist.id,
  46     videoId: video.id
  47   }
  48 }
  49
  50 async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) {
  51   await Bluebird.map(playlistUrls, async playlistUrl => {
  52     try {
  53       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  54       if (exists === true) return
  55
  56       // Fetch url
  57       const { body } = await doRequest<PlaylistObject>({
  58         uri: playlistUrl,
  59         json: true,
  60         activityPub: true
  61       })
  62
  63       if (!isPlaylistObjectValid(body)) {
  64         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  65       }
  66
  67       if (!isArray(body.to)) {
  68         throw new Error('Playlist does not have an audience.')
  69       }
  70
  71       return createOrUpdateVideoPlaylist(body, account, body.to)
  72     } catch (err) {
  73       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  74     }
  75   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  76 }
  77
  78 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModelId, to: string[]) {
  79   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  80
  81   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  82     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  83
  84     if (actor.VideoChannel) {
  85       playlistAttributes.videoChannelId = actor.VideoChannel.id
  86     } else {
  87       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  88     }
  89   }
  90
  91   const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true })
  92
  93   let accItems: string[] = []
  94   await crawlCollectionPage<string>(playlistObject.id, items => {
  95     accItems = accItems.concat(items)
  96
  97     return Promise.resolve()
  98   })
  99
 100   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
 101
 102   if (playlistObject.icon) {
 103     try {
 104       const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist)
 105       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 106     } catch (err) {
 107       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 108     }
 109   } else if (refreshedPlaylist.hasThumbnail()) {
 110     await refreshedPlaylist.Thumbnail.destroy()
 111     refreshedPlaylist.Thumbnail = null
 112   }
 113
 114   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 115 }
 116
 117 async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
 118   if (!videoPlaylist.isOutdated()) return videoPlaylist
 119
 120   try {
 121     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 122     if (statusCode === 404) {
 123       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 124
 125       await videoPlaylist.destroy()
 126       return undefined
 127     }
 128
 129     if (playlistObject === undefined) {
 130       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 131
 132       await videoPlaylist.setAsRefreshed()
 133       return videoPlaylist
 134     }
 135
 136     const byAccount = videoPlaylist.OwnerAccount
 137     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 138
 139     return videoPlaylist
 140   } catch (err) {
 141     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 142
 143     await videoPlaylist.setAsRefreshed()
 144     return videoPlaylist
 145   }
 146 }
 147
 148 // ---------------------------------------------------------------------------
 149
 150 export {
 151   createAccountPlaylists,
 152   playlistObjectToDBAttributes,
 153   playlistElementObjectToDBAttributes,
 154   createOrUpdateVideoPlaylist,
 155   refreshVideoPlaylistIfNeeded
 156 }
 157
 158 // ---------------------------------------------------------------------------
 159
 160 async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) {
 161   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 162
 163   await Bluebird.map(elementUrls, async elementUrl => {
 164     try {
 165       // Fetch url
 166       const { body } = await doRequest<PlaylistElementObject>({
 167         uri: elementUrl,
 168         json: true,
 169         activityPub: true
 170       })
 171
 172       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 173
 174       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 175         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 176       }
 177
 178       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 179
 180       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 181     } catch (err) {
 182       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 183     }
 184   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 185
 186   await sequelizeTypescript.transaction(async t => {
 187     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 188
 189     for (const element of elementsToCreate) {
 190       await VideoPlaylistElementModel.create(element, { transaction: t })
 191     }
 192   })
 193
 194   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 195
 196   return undefined
 197 }
 198
 199 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 200   const options = {
 201     uri: playlistUrl,
 202     method: 'GET',
 203     json: true,
 204     activityPub: true
 205   }
 206
 207   logger.info('Fetching remote playlist %s.', playlistUrl)
 208
 209   const { response, body } = await doRequest(options)
 210
 211   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 212     logger.debug('Remote video playlist JSON is not valid.', { body })
 213     return { statusCode: response.statusCode, playlistObject: undefined }
 214   }
 215
 216   return { statusCode: response.statusCode, playlistObject: body }
 217 }