server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
   4 import { isArray } from '../../helpers/custom-validators/misc'
   5 import { getOrCreateActorAndServerAndModel } from './actor'
   6 import { logger } from '../../helpers/logger'
   7 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   8 import { doRequest } from '../../helpers/requests'
   9 import { checkUrlsSameHost } from '../../helpers/activitypub'
  10 import * as Bluebird from 'bluebird'
  11 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  12 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  13 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  14 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  15 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  16 import { sequelizeTypescript } from '../../initializers/database'
  17 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  18 import { FilteredModelAttributes } from '../../typings/sequelize'
  19 import { MAccountDefault, MAccountId, MVideoId } from '../../typings/models'
  20 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../typings/models/video/video-playlist'
  21
  22 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  23   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
  24
  25   return {
  26     name: playlistObject.name,
  27     description: playlistObject.content,
  28     privacy,
  29     url: playlistObject.id,
  30     uuid: playlistObject.uuid,
  31     ownerAccountId: byAccount.id,
  32     videoChannelId: null,
  33     createdAt: new Date(playlistObject.published),
  34     updatedAt: new Date(playlistObject.updated)
  35   }
  36 }
  37
  38 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
  39   return {
  40     position: elementObject.position,
  41     url: elementObject.id,
  42     startTimestamp: elementObject.startTimestamp || null,
  43     stopTimestamp: elementObject.stopTimestamp || null,
  44     videoPlaylistId: videoPlaylist.id,
  45     videoId: video.id
  46   }
  47 }
  48
  49 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  50   await Bluebird.map(playlistUrls, async playlistUrl => {
  51     try {
  52       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  53       if (exists === true) return
  54
  55       // Fetch url
  56       const { body } = await doRequest<PlaylistObject>({
  57         uri: playlistUrl,
  58         json: true,
  59         activityPub: true
  60       })
  61
  62       if (!isPlaylistObjectValid(body)) {
  63         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  64       }
  65
  66       if (!isArray(body.to)) {
  67         throw new Error('Playlist does not have an audience.')
  68       }
  69
  70       return createOrUpdateVideoPlaylist(body, account, body.to)
  71     } catch (err) {
  72       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  73     }
  74   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  75 }
  76
  77 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  78   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  79
  80   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  81     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  82
  83     if (actor.VideoChannel) {
  84       playlistAttributes.videoChannelId = actor.VideoChannel.id
  85     } else {
  86       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  87     }
  88   }
  89
  90   const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  91
  92   let accItems: string[] = []
  93   await crawlCollectionPage<string>(playlistObject.id, items => {
  94     accItems = accItems.concat(items)
  95
  96     return Promise.resolve()
  97   })
  98
  99   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
 100
 101   if (playlistObject.icon) {
 102     try {
 103       const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist)
 104       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 105     } catch (err) {
 106       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 107     }
 108   } else if (refreshedPlaylist.hasThumbnail()) {
 109     await refreshedPlaylist.Thumbnail.destroy()
 110     refreshedPlaylist.Thumbnail = null
 111   }
 112
 113   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 114 }
 115
 116 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
 117   if (!videoPlaylist.isOutdated()) return videoPlaylist
 118
 119   try {
 120     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 121     if (statusCode === 404) {
 122       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 123
 124       await videoPlaylist.destroy()
 125       return undefined
 126     }
 127
 128     if (playlistObject === undefined) {
 129       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 130
 131       await videoPlaylist.setAsRefreshed()
 132       return videoPlaylist
 133     }
 134
 135     const byAccount = videoPlaylist.OwnerAccount
 136     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 137
 138     return videoPlaylist
 139   } catch (err) {
 140     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 141
 142     await videoPlaylist.setAsRefreshed()
 143     return videoPlaylist
 144   }
 145 }
 146
 147 // ---------------------------------------------------------------------------
 148
 149 export {
 150   createAccountPlaylists,
 151   playlistObjectToDBAttributes,
 152   playlistElementObjectToDBAttributes,
 153   createOrUpdateVideoPlaylist,
 154   refreshVideoPlaylistIfNeeded
 155 }
 156
 157 // ---------------------------------------------------------------------------
 158
 159 async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 160   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 161
 162   await Bluebird.map(elementUrls, async elementUrl => {
 163     try {
 164       // Fetch url
 165       const { body } = await doRequest<PlaylistElementObject>({
 166         uri: elementUrl,
 167         json: true,
 168         activityPub: true
 169       })
 170
 171       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 172
 173       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 174         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 175       }
 176
 177       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 178
 179       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 180     } catch (err) {
 181       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 182     }
 183   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 184
 185   await sequelizeTypescript.transaction(async t => {
 186     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 187
 188     for (const element of elementsToCreate) {
 189       await VideoPlaylistElementModel.create(element, { transaction: t })
 190     }
 191   })
 192
 193   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 194
 195   return undefined
 196 }
 197
 198 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 199   const options = {
 200     uri: playlistUrl,
 201     method: 'GET',
 202     json: true,
 203     activityPub: true
 204   }
 205
 206   logger.info('Fetching remote playlist %s.', playlistUrl)
 207
 208   const { response, body } = await doRequest(options)
 209
 210   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 211     logger.debug('Remote video playlist JSON is not valid.', { body })
 212     return { statusCode: response.statusCode, playlistObject: undefined }
 213   }
 214
 215   return { statusCode: response.statusCode, playlistObject: body }
 216 }