server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
   4 import { AccountModel } from '../../models/account/account'
   5 import { isArray } from '../../helpers/custom-validators/misc'
   6 import { getOrCreateActorAndServerAndModel } from './actor'
   7 import { logger } from '../../helpers/logger'
   8 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   9 import { doRequest } from '../../helpers/requests'
  10 import { checkUrlsSameHost } from '../../helpers/activitypub'
  11 import * as Bluebird from 'bluebird'
  12 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  13 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  14 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  15 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  16 import { VideoModel } from '../../models/video/video'
  17 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
  18 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  19 import { sequelizeTypescript } from '../../initializers/database'
  20 import { createPlaylistThumbnailFromUrl } from '../thumbnail'
  21
  22 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
  23   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
  24
  25   return {
  26     name: playlistObject.name,
  27     description: playlistObject.content,
  28     privacy,
  29     url: playlistObject.id,
  30     uuid: playlistObject.uuid,
  31     ownerAccountId: byAccount.id,
  32     videoChannelId: null,
  33     createdAt: new Date(playlistObject.published),
  34     updatedAt: new Date(playlistObject.updated)
  35   }
  36 }
  37
  38 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) {
  39   return {
  40     position: elementObject.position,
  41     url: elementObject.id,
  42     startTimestamp: elementObject.startTimestamp || null,
  43     stopTimestamp: elementObject.stopTimestamp || null,
  44     videoPlaylistId: videoPlaylist.id,
  45     videoId: video.id
  46   }
  47 }
  48
  49 async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) {
  50   await Bluebird.map(playlistUrls, async playlistUrl => {
  51     try {
  52       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  53       if (exists === true) return
  54
  55       // Fetch url
  56       const { body } = await doRequest<PlaylistObject>({
  57         uri: playlistUrl,
  58         json: true,
  59         activityPub: true
  60       })
  61
  62       if (!isPlaylistObjectValid(body)) {
  63         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  64       }
  65
  66       if (!isArray(body.to)) {
  67         throw new Error('Playlist does not have an audience.')
  68       }
  69
  70       return createOrUpdateVideoPlaylist(body, account, body.to)
  71     } catch (err) {
  72       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  73     }
  74   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  75 }
  76
  77 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
  78   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  79
  80   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  81     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  82
  83     if (actor.VideoChannel) {
  84       playlistAttributes.videoChannelId = actor.VideoChannel.id
  85     } else {
  86       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  87     }
  88   }
  89
  90   const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true })
  91
  92   let accItems: string[] = []
  93   await crawlCollectionPage<string>(playlistObject.id, items => {
  94     accItems = accItems.concat(items)
  95
  96     return Promise.resolve()
  97   })
  98
  99   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
 100
 101   if (playlistObject.icon) {
 102     try {
 103       const thumbnailModel = await createPlaylistThumbnailFromUrl(playlistObject.icon.url, refreshedPlaylist)
 104       thumbnailModel.videoPlaylistId = refreshedPlaylist.id
 105
 106       refreshedPlaylist.setThumbnail(await thumbnailModel.save())
 107     } catch (err) {
 108       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 109     }
 110   }
 111
 112   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 113 }
 114
 115 async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
 116   if (!videoPlaylist.isOutdated()) return videoPlaylist
 117
 118   try {
 119     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 120     if (statusCode === 404) {
 121       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 122
 123       await videoPlaylist.destroy()
 124       return undefined
 125     }
 126
 127     if (playlistObject === undefined) {
 128       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 129
 130       await videoPlaylist.setAsRefreshed()
 131       return videoPlaylist
 132     }
 133
 134     const byAccount = videoPlaylist.OwnerAccount
 135     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 136
 137     return videoPlaylist
 138   } catch (err) {
 139     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 140
 141     await videoPlaylist.setAsRefreshed()
 142     return videoPlaylist
 143   }
 144 }
 145
 146 // ---------------------------------------------------------------------------
 147
 148 export {
 149   createAccountPlaylists,
 150   playlistObjectToDBAttributes,
 151   playlistElementObjectToDBAttributes,
 152   createOrUpdateVideoPlaylist,
 153   refreshVideoPlaylistIfNeeded
 154 }
 155
 156 // ---------------------------------------------------------------------------
 157
 158 async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) {
 159   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 160
 161   await Bluebird.map(elementUrls, async elementUrl => {
 162     try {
 163       // Fetch url
 164       const { body } = await doRequest<PlaylistElementObject>({
 165         uri: elementUrl,
 166         json: true,
 167         activityPub: true
 168       })
 169
 170       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 171
 172       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 173         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 174       }
 175
 176       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 177
 178       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 179     } catch (err) {
 180       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 181     }
 182   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 183
 184   await sequelizeTypescript.transaction(async t => {
 185     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 186
 187     for (const element of elementsToCreate) {
 188       await VideoPlaylistElementModel.create(element, { transaction: t })
 189     }
 190   })
 191
 192   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 193
 194   return undefined
 195 }
 196
 197 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 198   const options = {
 199     uri: playlistUrl,
 200     method: 'GET',
 201     json: true,
 202     activityPub: true
 203   }
 204
 205   logger.info('Fetching remote playlist %s.', playlistUrl)
 206
 207   const { response, body } = await doRequest(options)
 208
 209   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 210     logger.debug('Remote video playlist JSON is not valid.', { body })
 211     return { statusCode: response.statusCode, playlistObject: undefined }
 212   }
 213
 214   return { statusCode: response.statusCode, playlistObject: body }
 215 }